package com.algo.url;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URL;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * multiple threads get title, key words, description from url list
 * @author lujianfeng@miaozhen.com
 */

public class TitleCrawler implements Runnable
{
	private static BufferedReader br;
	private static BufferedWriter bw;
	private static ScratchLogger logger;

    private static int socketTimeOutMillis = 3000;
    private static boolean reScratch = false;
    private static long scratchInterval = 0;
	public static class EmptyTitleException extends IOException{
		private static final long serialVersionUID = 1L;
	}
    public static void setSocketTimeOutMillis(int timeOutMillis){
        socketTimeOutMillis = timeOutMillis;
    }
    public static void setReScratch(boolean reScratch){
                TitleCrawler.reScratch = reScratch;
                    }
    public static void setScratchInterval(long interval){
            scratchInterval = interval;
        }
	
	public static void close(){
		try{
			if(br != null) br.close();
			if(bw != null)	bw.close();
			if(logger != null)	logger.closeLogger();
		}catch(IOException e){
			throw new RuntimeException(e);
		}
	}
	
	public static String getTitle(String urlLog) throws InterruptedException  {
		Document doc;
		
        String url;
        if(reScratch){
            int pos = urlLog.indexOf("^");
            String exceptionName = urlLog.substring(pos+1);
            if(exceptionName.contains("EmptyTitleException")) return null;
            else if(exceptionName.contains("HttpStatusException") && scratchInterval > 0)
                             Thread.sleep(scratchInterval);
            url = urlLog.substring(0, pos);
        }
        else url = urlLog;
                                
		try {
			new URL(url);
			Connection con = Jsoup.connect(url);
            con.timeout(socketTimeOutMillis);
			con.userAgent("Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)");
			doc = con.get();
		}catch (IOException e) {
			if(logger != null)	logger.logExeption(urlLog, e);
			return null;
		}
		
    	Elements heads = doc.getElementsByTag("head");
    	
    	StringBuilder sb = new StringBuilder();
    	int[] isEmpty = new int[3];
    	for(Element head : heads){
    		Elements titles = head.getElementsByTag("title");
    		for(Element title : titles){
    			sb.append(title.text().replaceAll("[\r\n\t]", " "));
    		}
    		if(sb.length() != 0)    isEmpty[0] = 1;
    		sb.append("\t");
    		Elements keys = head.getElementsByAttributeValue("name", "keywords");
    		for(Element key : keys){
    			sb.append(key.attr("content").replaceAll("[\r\n\t]", " "));
    		}
    		if(sb.charAt(sb.length() - 1) != '\t') isEmpty[1] = 1;
    		sb.append("\t");
    		Elements descs = head.getElementsByAttributeValue("name", "description");
    		for(Element desc : descs){
    			sb.append(desc.attr("content").replaceAll("[\r\n\t]", " "));
    		}
            if(sb.charAt(sb.length() - 1) != '\t') isEmpty[2] = 1;
    	}
    	
        String title =  sb.length() > 3 ? url + "\t" + sb.toString() + "\n" : null;
        if(logger != null){
	        if(title == null)	logger.logExeption(urlLog, new EmptyTitleException());
	        else logger.afterScratchUrl(isEmpty);
        }
        return title;
	}
	
	public TitleCrawler(String in, String out, String log){
		try {
			br = new BufferedReader(new FileReader(in));
			bw = new BufferedWriter(new FileWriter(out));
			logger = new ScratchLogger(log);
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	}
	
	
	
	public TitleCrawler(){}
	
	public void run(){
		while(true){
			try{
				String url = null;
				synchronized(br){
					url = br.readLine();
				}
				if(url == null)	break;
		    	String title = getTitle(url);
	    		if(title != null)
	    			synchronized(bw){
	    				bw.write(title);
	    			}
			}catch(IOException e){
				throw new RuntimeException(e);
			}catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
		}
	}
	
	public void multiThreadsGetTitle(int threadsNum, int timeOutDays) throws InterruptedException{
    	ExecutorService executor = Executors.newCachedThreadPool();
    	for(int i = 0; i < threadsNum; i++)
    		executor.execute(new TitleCrawler());
    	executor.shutdown();
    	System.out.println("INFO: " + threadsNum + " threads started at system time: "+ System.currentTimeMillis());
    	executor.awaitTermination(timeOutDays, TimeUnit.DAYS);
    	TitleCrawler.close();
    	System.out.println("INFO: " + threadsNum + " threads closed at system time: "+ System.currentTimeMillis());
	}
	
    public static void main( String[] args ) throws InterruptedException{
    
    	if(args.length >= 6)
            TitleCrawler.setSocketTimeOutMillis(Integer.parseInt(args[5]));
        if(args.length >= 7)
            TitleCrawler.setReScratch(Boolean.parseBoolean(args[6]));
        if(args.length >= 8) 
          TitleCrawler.setScratchInterval(Long.parseLong(args[7]));
    	new TitleCrawler(args[0], args[1], args[2]).multiThreadsGetTitle(Integer.parseInt(args[3]), Integer.parseInt(args[4]));
    	
    	//new TitleCrawler("D://java//url50", "D://java//title", "D://java//log").multiThreadsGetTitle(1, 2);
    	logger.printScratchInfo();
    }
	
}
